# Combining the raw AMPD csv files into one fst file 
library(pacman)
p_load(
  here, fst, data.table, janitor, purrr
)

# File path for the raw data
raw_fp = "Data/electricity-generation/cems/raw"

combine_cems = function(yr){
  # Read all of the csv's from one year
  raw_cems_yr = 
    map_dfr(
      list.files(here(raw_fp, yr)),
      \(file){fread(file = here(raw_fp, yr, file)) |> clean_names()}
    )
  # Save as an fst
  write.fst(
    raw_cems_yr, 
    path = here(paste0("Data/electricity-generation/cems/raw_cems_",yr,".fst"))
  )
  
  rm(raw_cems_yr)
  invisible(gc())
} 
    
# Running for all years
map(
  2018:2022,
  combine_cems
)

